# -*- coding:utf-8 -*-

import urllib
from BeautifulSoup import BeautifulSoup
import MySQLdb

arr = []

# Find all urls in one URL
def lookforurl(url):
    # Get page content
    content = urllib.urlopen(url).read()
    # Make it --> BeautifulSoup
    try:
        soup = BeautifulSoup(content)
    except Exception, e:
        return
    # Find A tag items
    for link in soup.findAll("a"):
        # Find href string
        linkString = str(link.get("href"))
        # Not None
        if linkString is not None:
            # Remove space
            linkStringNoSpace = linkString.strip()
            # re-makeup the link
            if linkStringNoSpace.startswith('/'):
                linkStringNoSpace = "http://www.21cake.com" + linkStringNoSpace
            if linkStringNoSpace.startswith('javascript'):
                linkStringNoSpace = ""
            if not linkStringNoSpace.startswith('http'):
                linkStringNoSpace = "http://www.21cake.com/" + linkStringNoSpace
            # Add link to array if new
            if linkStringNoSpace not in arr:
                arr.append(str(linkStringNoSpace))
                # For Debug
                print linkStringNoSpace
                lookforurl(linkStringNoSpace)

def savedb(url, code):
    sql = ""
    if code == 200:
        if '.jpg' in str(url):
            sql = "INSERT INTO `SITEMAPTEST`.`SiteURLCheck`(`sitepath`,`status`,`isimage`,`datetime`,`code`) VALUES ('{0}', 'Success', 1, now(), {1})".format(url, code)
        else:
            sql = "INSERT INTO `SITEMAPTEST`.`SiteURLCheck`(`sitepath`,`status`,`isimage`,`datetime`,`code`) VALUES ('{0}', 'Success', 0, now(), {1})".format(url, code)
    else:
        if '.jpg' in str(url):
            sql = "INSERT INTO `SITEMAPTEST`.`SiteURLCheck`(`sitepath`,`status`,`isimage`,`datetime`,`code`) VALUES ('{0}', 'Fail', 1, now(), {1})".format(url, code)
        else:
            sql = "INSERT INTO `SITEMAPTEST`.`SiteURLCheck`(`sitepath`,`status`,`isimage`,`datetime`,`code`) VALUES ('{0}', 'Fail', 0, now(), {1})".format(url, code)

    db = MySQLdb.connect(user="root", passwd="1", host="127.0.0.1", db="SITEMAPTEST", port=3306)
    cursor = db.cursor()
    cursor.execute(sql)
    db.commit()



def checkurl(url):

    statusCode = urllib.urlopen(url).getcode()
    print "URL: " + url + " Code: " + str(statusCode)
    savedb(url, statusCode)


# Main Enter
lookforurl("http://www.21cake.com")

for item in arr:
    checkurl(str(item))